//Tic-Tac-Toe Artificial Intelligence
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.util.*;

//Created: October 13, 2008
import javax.swing.JButton;

public class QLearningAgent	{

	final double alpha = 0.2; // Learning rate
	final double gamma = 0.9; // Discount Factor
	final double epsilon = 0.8;// exploration rate
	String currState;
	ArrayList<String> possibleStates;
	ArrayList<Integer> legalActions;
	int bestAction;
	HashMap<String,Double> qFunc;

	public QLearningAgent(JButton [] buttons){
		currState = toState(buttons);
		qFunc = new HashMap<String, Double>();
	}

	public void updateQ(String prev,JButton[] curr, int reward){
		String currs = toVector(toState(curr));
		prev = toVector(prev);
		double OldQ = qFunc.get(prev);
		double newQ = qFunc.get(currs);
		qFunc.put(prev, (1-alpha)*OldQ+alpha*(reward+gamma*newQ));
	}


	public void updateQ(String prev, int reward){
		String v = toVector(prev);
		double OldQ = qFunc.get(v);
		qFunc.put(v, (1-alpha)*OldQ+alpha*(reward));
	}

	public void reset(JButton[] buttons){
		currState = toState(buttons);
	}

	public int getBestAction(JButton[] buttons)	{
		currState = toState(buttons);
		legalActions = getLegalActions();
		possibleStates = generatePossibleStates();

		double MAX = -10000;
		int indexOfPolicy = 0;
		for(int i = 0; i<legalActions.size();i++){
			String v = toVector(possibleStates.get(i));
			if(!qFunc.containsKey(v))	qFunc.put(v, (double)0);
			double Qvalue = qFunc.get(v);
			if(Qvalue>MAX){
				MAX = Qvalue;
				indexOfPolicy = legalActions.get(i);
			}
		}
		if(Math.random()>epsilon){
			if(legalActions.size()!=0)
			indexOfPolicy = legalActions.get((int)Math.random()*legalActions.size());
			
		}
		return indexOfPolicy;
	}

	public void printQ() throws IOException{
		Set<String> s = qFunc.keySet();
		BufferedWriter writer = new BufferedWriter(new FileWriter("test.txt"));
		for(String str:s){
			writer.write(str+" "+qFunc.get(str)+"\n");
		}
		 writer.close();
	}

	public ArrayList<String> generatePossibleStates(){
		ArrayList<String> tempStates = new ArrayList<String>();
		for(int i = 0; i<legalActions.size();i++){
			int index = legalActions.get(i);
			String tmp = currState.substring(0, index)+"1"+currState.substring(index+1);
			tempStates.add(tmp);
		}
		return tempStates;
	}
	public ArrayList<Integer> getLegalActions(){
		ArrayList<Integer> tmp = new ArrayList<Integer>();

		//get legal actions given a state
		for(int i = 0; i< currState.length(); i++){
			if(currState.charAt(i)== '0'){
				tmp.add(i);
			}
		}
		return tmp;
	}
	private static String toNum(String[] state){
		String num = "";
		for(int i=0; i<state.length; i++){
			if (state[i]==""){
				num += "0";
			}
			else if (state[i]=="X"){
				num += "1";
			}
			else{
				num += "2";
			}
		}
		return num;
	}
	public String toState(JButton [] buttons){

		String[] tmpState = new String[buttons.length-1];
		for(int i = 0; i< buttons.length-1; i++){

			tmpState[i] = buttons[i+1].getText();
		}
		return toNum(tmpState);
	}

	private static int isCenter(String s){
		// blank - 0; X - 1; O - 2
		if(s.charAt(4)== '1'){
			return 0;
		}else if(s.charAt(4)== '2'){
			return 1;
		}
		return 2;
	}

	private static int isWinning(String s){
		char[] tmp = s.toCharArray();
		if(tmp[0] == tmp[1] && tmp[1] == tmp[2]&&tmp[0]!='0')	return tmp[0]=='1'?0:1;
		if(tmp[3] == tmp[4] && tmp[4] == tmp[5]&&tmp[3]!='0')	return tmp[3]=='1'?0:1;
		if(tmp[6] == tmp[7] && tmp[7] == tmp[8]&&tmp[6]!='0')	return tmp[6]=='1'?0:1;
		if(tmp[0] == tmp[3] && tmp[3] == tmp[6]&&tmp[0]!='0')	return tmp[0]=='1'?0:1;
		if(tmp[1] == tmp[4] && tmp[4] == tmp[7]&&tmp[1]!='0')	return tmp[1]=='1'?0:1;
		if(tmp[2] == tmp[5] && tmp[5] == tmp[8]&&tmp[2]!='0')	return tmp[2]=='1'?0:1;
		if(tmp[0] == tmp[4] && tmp[4] == tmp[8]&&tmp[0]!='0')	return tmp[0]=='1'?0:1;
		if(tmp[2] == tmp[4] && tmp[4] == tmp[6]&&tmp[2]!='0')	return tmp[2]=='1'?0:1;
		return 2;
	}

	private static int isAboutToWin(String s){
		int[] tmp = new int[9];
		int x = 0, o = 0;
		for(int i = 0; i<s.length(); i++){
			if(s.charAt(i) == '1')	tmp[i] = 1;
			else if(s.charAt(i) == '2')	tmp[i] = -1;
			else tmp[i] = 0;
		}
		if(tmp[0]+tmp[1]+tmp[2]==2)	x++;else if(tmp[0]+tmp[1]+tmp[2]==-2)	o++;
		if(tmp[3]+tmp[4]+tmp[5]==2)	x++;else if(tmp[3]+tmp[4]+tmp[5]==-2)	o++;
		if(tmp[6]+tmp[7]+tmp[8]==2)	x++;else if(tmp[6]+tmp[7]+tmp[8]==-2)	o++;
		if(tmp[0]+tmp[3]+tmp[6]==2)	x++;else if(tmp[0]+tmp[3]+tmp[6]==-2)	o++;
		if(tmp[1]+tmp[4]+tmp[7]==2)	x++;else if(tmp[1]+tmp[4]+tmp[7]==-2)	o++;
		if(tmp[2]+tmp[5]+tmp[8]==2)	x++;else if(tmp[2]+tmp[5]+tmp[8]==-2)	o++;
		if(tmp[0]+tmp[4]+tmp[8]==2)	x++;else if(tmp[0]+tmp[4]+tmp[8]==-2)	o++;
		if(tmp[2]+tmp[4]+tmp[6]==2)	x++;else if(tmp[2]+tmp[4]+tmp[6]==-2)	o++;

		if(o==x) return 0;
		else if(o<x) return 1;
		return 2;
	}
	private static int numOfCell(String s){
		int num = 0;
		for(int i=0; i<s.length(); i++){
			if(s.charAt(i) != '0'){
				num++;
			}
		}

		if(0<= num && num <=3){ return 0; }
		else if (4<=num && num <= 6) {return 1;}
		else {return 2;}
	}

	private static int isCorner(String s){
		// 0 - blank, 1 - X, 2 - O
		// 0 2 6 8 A - num coners occupied by X, B - num corners occupied by O
		int A = 0;
		int B = 0;
		if(s.charAt(0) == '1') { A++; }
		else if(s.charAt(0) == '2'){ B++; }

		if(s.charAt(2) == '1'){    A++; }
		else if(s.charAt(2) == '2'){ B++; }

		if(s.charAt(6) == '1'){    A++; }
		else if(s.charAt(6) == '2'){ B++; }

		if(s.charAt(8) == '1'){    A++; }
		else if(s.charAt(8) == '2'){ B++; }

		if (A==B){ return 0; }
		else if (A>B){ return 1; }
		else{ return 2; }
	}

	private static int isSide(String s){
		// 1 3 5 7
		int A = 0;
		int B = 0;
		if(s.charAt(1) == '1') { A++; }
		else if(s.charAt(1) == '2'){ B++; }

		if(s.charAt(3) == '1'){    A++; }
		else if(s.charAt(3) == '2'){ B++; }

		if(s.charAt(5) == '1'){    A++; }
		else if(s.charAt(5) == '2'){ B++; }

		if(s.charAt(7) == '1'){    A++; }
		else if(s.charAt(7) == '2'){ B++; }

		if (A==B){ return 0; }
		else if (A>B){ return 1; }
		else{ return 2; }
	}
	private static String toVector(String s){
		int isCenter = isCenter(s);
		int isWinning = isWinning(s);
		int isAboutToWin = isAboutToWin(s);
		int isSide = isSide(s);
		int isCorner = isCorner(s);
		int numOfCell = numOfCell(s);
		return String.valueOf(isCenter)+String.valueOf(isWinning)+String.valueOf(isAboutToWin)
				+String.valueOf(numOfCell)+String.valueOf(isCorner)+String.valueOf(isSide);
	}
	
	

}